# coding=utf8
import requests
import time
from bs4 import BeautifulSoup
from libs.sele import driver_new_session
from models.tw import model_tw_user


def get_profile(driver, user_list):
    span_map = {
        0: 'Twitters', 1: 'Following', 2: 'Followers', 3: 'Favorites'
    }
    for user in user_list:
        print(user['Name'])
        if model_tw_user.find_one({'HomePage': user['HomePage']}) is not None:
            continue
        driver.get(user['HomePage'])
        if driver.title == 'Twitter / ?':
            print("用户不存在")
            continue
        for idx, span in enumerate(driver.find_elements_by_css_selector('.ProfileNav-list li span:nth-child(2)')):
            if idx > 3:
                break
            user.update({span_map[idx]: span.get_attribute('innerHTML')})
        yield user
        time.sleep(5)


if __name__ == '__main__':
    # 获取用户list
    user_list = []
    # 使用代理
    proxy = {"http": "socks5://127.0.0.1:8080", "https": "socks5://127.0.0.1:8080"}
    soup = BeautifulSoup(requests.get('http://www.twibes.com/gay/twitter-list', proxies=proxy).text, 'lxml')
    for div in soup.select('div.user'):
        user_name = div.find('a')['href'].split('/')[2]
        user_list.append({
            'Name': user_name,
            'HomePage': 'https://twitter.com/' + user_name
        })

    driver = driver_new_session()
    for user in get_profile(driver, user_list):
        model_tw_user.insert_one(user)
